import scrapy


class BqgSpider(scrapy.Spider):
    name = 'bqg'
    allowed_domains = ['fyrsks.com']
    start_urls = ['http://www.fyrsks.com/bqg/3429/4732578.html']
    # start_urls = ['http://www.fyrsks.com/bqg/3429/5635201.html']

    def parse(self, response):
        txt_name = response.xpath('//h1/text()').get()
        txt_content = response.xpath('string(//div[@id ="content"])').get()
        # txt_content = ''
        # for t in txt_contents:
        #     txt_content += t
        txt_name = txt_name.replace('?', '')
        txt_name = txt_name.replace('!', '')
        txt_content = txt_content.replace(" 章节错误,点此举报(免注册),举报后维护人员会在两分钟内校正章节内容,请耐心等待,并刷新页面。", " ")
        txt_content = txt_content.replace(" ", "")
        txt_content = txt_content.replace("\r\n", "")
        txt_content = txt_content.replace("\u3000\u3000\xa0\u3000\u3000", "\r\n")
        txt_content = txt_content.replace("\u3000\u3000\u3000\u3000", "\r\n")

        yield {
            'txt_name': txt_name,
            'txt_content': txt_content,
        }
        next_url = response.xpath("//div[@class='section-opt']/a[3]/@href").get()
        yield scrapy.Request(response.urljoin(next_url))

